
package net.bwie.realtime.jtp.dwd.job;


import com.alibaba.fastjson.JSON;
import net.bwie.realtime.jtp.commp.utils.KafkaUtil;
import net.bwie.realtime.jtp.dwd.function.AppLogSplitProcessFunction;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SideOutputDataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.util.Collector;
import org.apache.flink.util.OutputTag;

/**
 * DWD层数据应用开发：将ODS层采集原始日志数据，进行分类处理，存储Kafka队列。
 *      数据流向：kafka -> flink datastream -> kafka
 * @author xuanyu
 * @date 2025/5/18
 */
public class JtpLogEtlJob {

    public static void main(String[] args) throws Exception{
        // 1.执行环境-env
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);

        // 2.数据源-source
        DataStream<String> kafkaDataStream = KafkaUtil.consumerKafka(env, "topic-log");
        kafkaDataStream.print("kafka");

//        // 3.数据转换-transformation
        DataStream<String> pageStream = processLog(kafkaDataStream);

//        // 4.数据输出-sink
        KafkaUtil.producerKafka(pageStream, "dwd-traffic-page-log");

        // 5.触发执行-execute
        env.execute("JtpLogEtlJob") ;
    }


    /**
     * 对实时获取APP流量日志数据进行ETL处理，并且进行分流
     *      1-数据清洗
     *      2-新老访客状态标记修复
     *      3-数据分流
     */
    private static DataStream<String> processLog(DataStream<String> stream) {
        DataStream<String>jsonDataStream = applogCleaned(stream);    

        DataStream<String> pageStream = splitStream(jsonDataStream);
        
        return pageStream;
    }
    /**
     * 1-数据清洗
     */
    private static DataStream<String> logCleaned(DataStream<String> stream) {return null;}

    /**
     * 2-新老访客状态标记修复
     */
    private static DataStream<String> processIsNew(DataStream<String> stream) {return null;}

    /**
     * 3-数据分流
     */
    private static DataStream<String> splitStream(DataStream<String> stream) {
        final OutputTag<String>errorTag = new OutputTag<String>("error-log"){};
        final OutputTag<String> startTag = new OutputTag<String>("start-log"){};
        final OutputTag<String> displayTag = new OutputTag<String>("display-log"){};
        final OutputTag<String> actionTag = new OutputTag<String>("action-log"){};

        SingleOutputStreamOperator<String>pageStream=stream.process(
                new AppLogSplitProcessFunction(errorTag,startTag,displayTag,actionTag)
        );
        return pageStream;
    }
    private static DataStream<String> applogCleaned(DataStream<String> stream) {
        final OutputTag<String>errorTag = new OutputTag<String>("dirty-log"){};
        SingleOutputStreamOperator<String>cleanedStream=stream.process(
                new ProcessFunction<String, String>() {
                    @Override
                    public void processElement(String value, Context ctx, Collector<String> out) throws Exception {
                        try {
                            JSON.parseObject(value);
                            out.collect(value);
                        } catch (Exception e) {
                            ctx.output(errorTag,value);
                        }
                    }
                }
        );

        SideOutputDataStream<String>dirtyStream=cleanedStream.getSideOutput(errorTag);
        KafkaUtil.producerKafka(dirtyStream,"dwd-traffic-dirty-log");

        return cleanedStream;
    }


}